// Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
// Project developers.  See the top-level LICENSE file for dates and other
// details.  No copyright assignment is required to contribute to VisIt.

// ************************************************************************* //
//                            avtHDFSFileFormat.C                           //
// ************************************************************************* //

#include <avtHDFSFileFormat.h>

#include <map>
#include <string>
#include <vector>

#include <vtkCellData.h>
#include <vtkDoubleArray.h>
#include <vtkFloatArray.h>
#include <vtkPointData.h>
#include <vtkType.h>
#include <vtkUnsignedCharArray.h>
#include <vtkUnstructuredGrid.h>

#include <avtDatabaseMetaData.h>
#include <avtMaterial.h>

#include <Expression.h>

#include <FileFunctions.h>
#include <InvalidVariableException.h>
#include <InvalidDBTypeException.h>
#include <InvalidTimeStepException.h>

#include "visit_gzstream.h"

#include <dirent.h>
#include <stdarg.h>

using std::map;
using std::string;
using std::vector;

static void
strrepl(char *s, char orig, char repl)
{
    int i, n;
    if (!s) return;
    n = strlen(s);
    for (i = 0; i < n; i++)
        if (s[i] == orig) s[i] = repl;
}

static int
sscanhdfs(char *s, char const *fmt, ...)
{
    int n;
    va_list ap;

    va_start(ap, fmt);
    strrepl(s, ',', ' ');
    //strrepl(fmt, ',', ' ');
    n = vsscanf(s, fmt, ap);
    va_end(ap);
    return n;
}

// ****************************************************************************
//  Method: avtHDFSFileFormat constructor
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************

avtHDFSFileFormat::avtHDFSFileFormat(const char *fn)
    : avtMTMDFileFormat(FileFunctions::Dirname(fn))
{
    int sval;
    FileFunctions::VisItStat_t sbuf;
    char tmp[512];
    string dBDirName, baseName;

    // Should have passed in the name of a states.txt.gz file
    dBDirName = FileFunctions::Dirname(string(fn));
    baseName = FileFunctions::Basename(string(fn));

    if (baseName != "states.txt.gz")
    {
        EXCEPTION1(InvalidDBTypeException, "HDFS must open a states.txt.gz file");
    }

    // confirm ../users.txt.gz is one dir above dBDirName
    snprintf(tmp, sizeof(tmp), "%s/../../users.txt.gz", dBDirName.c_str());
    sval = FileFunctions::VisItStat(tmp, &sbuf);
    if (sval != 0 || !(sbuf.st_mode&S_IFREG))
    {
        snprintf(tmp, sizeof(tmp), "Unable to stat %s/../users.txt.gz", dBDirName.c_str());
        EXCEPTION1(InvalidDBTypeException, tmp);
    }

    // confirm ./dbs.txt.gz is in same dir as dBDirName 
    snprintf(tmp, sizeof(tmp), "%s/../dbs.txt.gz", dBDirName.c_str());
    sval = FileFunctions::VisItStat(tmp, &sbuf);
    if (sval != 0 || !(sbuf.st_mode&S_IFREG))
    {
        snprintf(tmp, sizeof(tmp), "Unable to stat %s/./dbs.txt.gz", dBDirName.c_str());
        EXCEPTION1(InvalidDBTypeException, tmp);
    }

    // confirm states.txt.gz is one dir below dBDirName
    snprintf(tmp, sizeof(tmp), "%s/states.txt.gz", dBDirName.c_str());
    sval = FileFunctions::VisItStat(tmp, &sbuf);
    if (sval != 0 || !(sbuf.st_mode&S_IFREG))
    {
        snprintf(tmp, sizeof(tmp), "unable to stat %s/states.txt.gz", dBDirName.c_str());
        EXCEPTION1(InvalidDBTypeException, tmp);
    }

    // confirm meshes.txt.gz is two dirs below dBDirName
    snprintf(tmp, sizeof(tmp), "%s/000000/meshes.txt.gz", dBDirName.c_str());
    sval = FileFunctions::VisItStat(tmp, &sbuf);
    if (sval != 0 || !(sbuf.st_mode&S_IFREG))
    {
        snprintf(tmp, sizeof(tmp), "unable to stat %s/000000/meshes.txt.gz", dBDirName.c_str());
        EXCEPTION1(InvalidDBTypeException, tmp);
    }
    
}

// ****************************************************************************
//  Method: avtHDFSFileFormat::GetNTimesteps
//
//  Purpose:
//      Tells the rest of the code how many timesteps there are in this file.
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************

static int CountLinesInFile(char const *dirname, char const *fname)
{
    char tmp[256];
    int n = 0;

    snprintf(tmp, sizeof(tmp), "%s/%s", dirname, fname);
    visit_ifstream ifile(tmp);
    while (!ifile().eof())
    {
        ifile().getline(tmp, sizeof(tmp));
        if (ifile().eof()) break;
        n++;
    }
    return n;
}

int
avtHDFSFileFormat::GetNTimesteps(void)
{
    return CountLinesInFile(filename, "states.txt.gz");
}

void
avtHDFSFileFormat::GetCycles(vector<int> &cycles)
{
    char tmp[256], key[32];
    int cycle, index;
    float time;
    snprintf(tmp, sizeof(tmp), "%s/states.txt.gz", filename);
    visit_ifstream ifile(tmp);
    while (!ifile().eof())
    {
        ifile().getline(tmp, sizeof(tmp));
        if (ifile().eof()) break;
        sscanhdfs(tmp, "%s %d %g %d", key, &index, &time, &cycle);
        cycles.push_back(cycle); 
    }
}

void
avtHDFSFileFormat::GetTimes(vector<double> &times)
{
    char tmp[256], key[32];
    int cycle, index;
    float time;
    snprintf(tmp, sizeof(tmp), "%s/states.txt.gz", filename);
    visit_ifstream ifile(tmp);
    while (!ifile().eof())
    {
        ifile().getline(tmp, sizeof(tmp));
        if (ifile().eof()) break;
        sscanhdfs(tmp, "%s %d %g %d", key, &index, &time, &cycle);
        times.push_back(time); 
    }
}

// ****************************************************************************
//  Method: avtHDFSFileFormat::FreeUpResources
//
//  Purpose:
//      When VisIt is done focusing on a particular timestep, it asks that
//      timestep to free up any resources (memory, file descriptors) that
//      it has associated with it.  This method is the mechanism for doing
//      that.
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************

void
avtHDFSFileFormat::FreeUpResources(void)
{
}

// ****************************************************************************
//  Method: avtHDFSFileFormat::PopulateDatabaseMetaData
//
//  Purpose:
//      This database meta-data object is like a table of contents for the
//      file.  By populating it, you are telling the rest of VisIt what
//      information it can request from you.
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************

void
avtHDFSFileFormat::PopulateDatabaseMetaData(avtDatabaseMetaData *md, int timeState)
{
    int nTimesteps = GetNTimesteps();

    if (timeState < 0 || timeState >= nTimesteps)
    {
        EXCEPTION2(InvalidTimeStepException, timeState, nTimesteps);
    }

    // users, dbs, states, meshes, blocks, leaf files

    // open the dir for the given timestate
    int sval;
    char tmp[512];
    FileFunctions::VisItStat_t sbuf;
    snprintf(tmp, sizeof(tmp), "%s/%06d/meshes.txt.gz", filename, timeState);
    sval = FileFunctions::VisItStat(tmp, &sbuf);
    if (sval != 0 || !(sbuf.st_mode&S_IFREG))
    {
        snprintf(tmp, sizeof(tmp), "Unable to stat %s/%06d/meshes.txt.gz", filename, timeState);
        EXCEPTION1(InvalidDBTypeException, tmp);
    }

    visit_ifstream ifile(tmp);
    while (!ifile().eof())
    {
        char line[256], mkey[32], mname[128];
        int nblocks, sdims, tdims;

        ifile().getline(line, sizeof(line));
        if (ifile().eof()) break;
        sscanhdfs(line, "%s %s %d %d %d", mkey, mname, &nblocks, &sdims, &tdims);

        if (nblocks == sdims == tdims == 1)
        {
            avtCurveMetaData *cmd = new avtCurveMetaData(mname);
            md->Add(cmd);
        }
        else
        {
            AddMeshToMetaData(md, mname, AVT_UNSTRUCTURED_MESH, NULL, nblocks, 0, sdims, tdims);
            avtLabelMetaData *lmdz = new avtLabelMetaData("zoneKeys", mname, AVT_ZONECENT);
            md->Add(lmdz);
            avtLabelMetaData *lmdn = new avtLabelMetaData("nodeKeys", mname, AVT_NODECENT);
            md->Add(lmdn);
        }

        // Use block 0 to determine variables
        snprintf(tmp, sizeof(tmp), "%s/%06d/%s/000000/variables.txt.gz", filename, timeState, mname);
        visit_ifstream vfile(tmp);
        while (!vfile().eof())
        {
            char vname[128];
            int cent, dtyp, ncomps;

            vfile().getline(line, sizeof(line));
            if (vfile().eof()) break;
            sscanhdfs(line, "%s %d %d %d", vname, &cent, &dtyp, &ncomps);

            int varInfo = cent<<16 | dtyp<<8 | ncomps;
            string varKey = string(mname) + ":" + string(vname);
            varInfoMap[varKey] = varInfo;

            avtCentering avcent = cent==0?AVT_NODECENT:AVT_ZONECENT;
            if (string(vname) == "materials")
                AddMaterialToMetaData(md, "material", mname, ncomps);
            else if (ncomps == 1)
                AddScalarVarToMetaData(md, vname, mname, avcent);
            else if (ncomps == sdims)
                AddVectorVarToMetaData(md, vname, mname, avcent, ncomps);
            else if (ncomps == sdims*sdims/2+1)
                AddSymmetricTensorVarToMetaData(md, vname, mname, avcent, ncomps);
            else if (ncomps == sdims*sdims)
                AddTensorVarToMetaData(md, vname, mname, avcent, ncomps);
            else
                AddArrayVarToMetaData(md, vname, ncomps, mname, avcent);
        }
        vfile.close();
    }
}


avtMaterial *
avtHDFSFileFormat::GetMaterial(int tim, int dom, char const *mname)
{
    // read topology data first (to get # of zones and keys for zones)
    char tmp[512], line[512], zkey[32];
    snprintf(tmp, sizeof(tmp), "%s/%06d/%s/%06d/topology.txt.gz",
        filename, tim, mname, dom);
    map<string, int> zoneKeyMap;
    int nzones = 0;
    visit_ifstream tfile(tmp);
    while (!tfile().eof())
    {
        tfile().getline(line, sizeof(line));
        if (tfile().eof()) break;
        sscanhdfs(line, "%s", zkey);
        zoneKeyMap[zkey] = nzones++;
    }
    tfile.close();

    // Read first line of materials to get material count
    int i, nmats = 0;
    char *p = &line[0];
    snprintf(tmp, sizeof(tmp), "%s/%06d/%s/%06d/materials.txt.gz",
        filename, tim, mname, dom);
    visit_ifstream matfile(tmp);
    matfile().getline(line, sizeof(line));
    while ((p = strchr(p, ',')) != NULL)
    {
        p++;
        nmats++;
    }

    // allocate things we'll need to build material object
    int *matnos = new int[nmats];
    char **matnames = new char*[nmats];
    float **vfracs = new float*[nmats];
    for (i = 0; i < nmats; i++)
    {
        vfracs[i] = new float[nzones];
        matnos[i] = i;
        snprintf(tmp, sizeof(tmp), "%d", i);
        matnames[i] = strdup(tmp);
    }

    // Read material file storing volume fractions
    while (!matfile().eof())
    {
        p = line;
        sscanhdfs(p, "%s", zkey); // changes ',' to ' '
        int zoneIdx = zoneKeyMap[zkey];
        p = strchr(p, ' ')+1;
        i = 0;
        while (i < nmats)
        {
            sscanf(p, "%f", &vfracs[i][zoneIdx]);
            i++;
            p = strchr(p, ' ')+1;
        }
        // getline at bottom of loop because first line was used
        // above to get material count
        matfile().getline(line, sizeof(line));
    }
    matfile.close();

    // construct the material object
    char domName[256];
    snprintf(domName, sizeof(domName), "%d", dom);
    avtMaterial *mat = new avtMaterial(nmats, matnos, matnames, 1, &nzones, 0, vfracs, domName);

    for (i = 0; i < nmats; i++)
    {
        delete [] vfracs[i];
        free(matnames[i]);
    }
    delete [] matnos;
    delete [] vfracs;
    delete [] matnames;

    return mat;
}

void *
avtHDFSFileFormat::GetAuxiliaryData(const char *var, int timestep, 
    int domain, const char *type, void *args, DestructorFunction &df)
{

    void *rv = NULL;

    if (strcmp(type, AUXILIARY_DATA_MATERIAL) == 0)
    {
        rv = (void *) GetMaterial(timestep, domain, metadata->MeshForVar(var).c_str());
        df = avtMaterial::Destruct;
    }

    return rv;
}

// ****************************************************************************
//  Method: avtHDFSFileFormat::GetMesh
//
//  Purpose:
//      Gets the mesh associated with this file.  The mesh is returned as a
//      derived type of vtkDataSet (ie vtkRectilinearGrid, vtkStructuredGrid,
//      vtkUnstructuredGrid, etc).
//
//  Arguments:
//      timestate   The index of the timestate.  If GetNTimesteps returned
//                  'N' time steps, this is guaranteed to be between 0 and N-1.
//      domain      The index of the domain.  If there are NDomains, this
//                  value is guaranteed to be between 0 and NDomains-1,
//                  regardless of block origin.
//      meshname    The name of the mesh of interest.  This can be ignored if
//                  there is only one mesh.
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************

vtkDataSet *
avtHDFSFileFormat::GetMesh(int timestate, int domain, const char *meshname)
{
    // open the coords.txt.gz file and read it
    char line[256];
    char key[32];
    double c[3];
    int ghost;
    bool hasGhostNodes = false;
    map<string, int> nodeKeyMap;
    vtkPoints *points = vtkPoints::New();
    vtkUnsignedCharArray *ghostNodes = vtkUnsignedCharArray::New();
    char tmp[512];
    snprintf(tmp, sizeof(tmp), "%s/%06d/%s/%06d/coords.txt.gz",
        filename, timestate, meshname, domain);
    visit_ifstream cfile(tmp);
    int i = 0;
    while (!cfile().eof())
    {
        cfile().getline(line, sizeof(line));
        if (cfile().eof()) break;
        sscanhdfs(line, "%s %d %lg %lg %lg", key, &ghost, &c[0], &c[1], &c[2]);
        points->InsertNextPoint(c[0],c[1],c[2]);
        nodeKeyMap[string(key)] = i++;
        ghostNodes->InsertNextTuple1((unsigned char) ghost);
        if (ghost) hasGhostNodes = true;
    }
    cfile.close();

    vtkUnstructuredGrid *ugrid = vtkUnstructuredGrid::New();
    ugrid->SetPoints(points);
    points->Delete();

    if (hasGhostNodes)
    {
        ghostNodes->SetName("avtGhostNodes");
        ugrid->GetPointData()->AddArray(ghostNodes);
    }
    ghostNodes->Delete();

    bool hasGhostZones = false;
    vtkUnsignedCharArray *ghostZones = vtkUnsignedCharArray::New();
    snprintf(tmp, sizeof(tmp), "%s/%06d/%s/%06d/topology.txt.gz",
        filename, timestate, meshname, domain);
    visit_ifstream tfile(tmp);
    char *p;
    int cnt, type;
    while (!tfile().eof())
    {
        tfile().getline(line, sizeof(line));
        if (tfile().eof()) break;
        sscanhdfs(line, "%s %d %d %d", key, &ghost, &type, &cnt);
        p = line;
        p = strchr(p, ' ')+1;
        p = strchr(p, ' ')+1;
        p = strchr(p, ' ')+1;
        p = strchr(p, ' ')+1;
        vtkIdType ids[64];
        for (i = 0; i < cnt; i++)
        {
            sscanf(p, "%s", key);
            p = strchr(p, ' ')+1;
            ids[i] = (vtkIdType) nodeKeyMap[string(key)];
        }
        ugrid->InsertNextCell(type, cnt, ids);
        ghostZones->InsertNextTuple1((unsigned char) ghost);
        if (ghost) hasGhostZones = true;
    }
    tfile.close();

    if (hasGhostZones)
    {
        ghostZones->SetName("avtGhostZones");
        ugrid->GetCellData()->AddArray(ghostZones);
    }
    ghostZones->Delete();

    return ugrid;
}

// ****************************************************************************
//  Method: avtHDFSFileFormat::GetVar
//
//  Purpose:
//      Gets a scalar variable associated with this file.  Although VTK has
//      support for many different types, the best bet is vtkFloatArray, since
//      that is supported everywhere through VisIt.
//
//  Arguments:
//      timestate  The index of the timestate.  If GetNTimesteps returned
//                 'N' time steps, this is guaranteed to be between 0 and N-1.
//      domain     The index of the domain.  If there are NDomains, this
//                 value is guaranteed to be between 0 and NDomains-1,
//                 regardless of block origin.
//      varname    The name of the variable requested.
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************
vtkDataArray *
avtHDFSFileFormat::GetVar(int timestate, int domain, const char *varname)
{
    int varInfo = 0x0;

    if (string(varname) == "zoneKeys")
        varInfo = AVT_ZONECENT<<16 | VTK_CHAR<<8 | 18;
    else if (string(varname) == "nodeKeys")
        varInfo = AVT_NODECENT<<16 | VTK_CHAR<<8 | 18;
    else
    {
        string varKey = metadata->MeshForVar(varname) + ":" + string(varname);
        if (varInfoMap.find(varKey) == varInfoMap.end())
            return 0;
        varInfo = varInfoMap[varKey];
    }

    int cent = (varInfo&0x00FF0000)>>16;
    int dtyp = (varInfo&0x0000FF00)>>8;
    int ncomps = (varInfo&0x000000FF);

    avtCentering avcent = (avtCentering) cent;

    char tmp[512];
    snprintf(tmp, sizeof(tmp), "%s/%06d/%s/%06d/%s.txt.gz",
        filename, timestate, metadata->MeshForVar(varname).c_str(), domain,
        avcent == AVT_NODECENT ? "coords" : "topology");

    int nents = 0;
    map<string, int> eKeyMap;
    visit_ifstream efile;

    char line[256], ekey[32];
    efile.open(tmp);
    while (!efile().eof())
    {
        efile().getline(line, sizeof(line));
        if (efile().eof()) break;
        sscanhdfs(line, "%s", ekey);
        eKeyMap[ekey] = nents++;
    }
    efile.close();

    vtkDataArray *darr = vtkDataArray::CreateDataArray(dtyp);
    darr->SetNumberOfComponents(ncomps);
    darr->SetNumberOfTuples(nents);

    if (string(varname) == "zoneKeys" || string(varname) == "nodeKeys")
    {
        for (map<string, int>::const_iterator it = eKeyMap.begin(); it != eKeyMap.end(); it++)
        {
            int eIdx = it->second;
            for (int i = 0; i < ncomps; i++)
                darr->SetComponent(eIdx, i, it->first[i]);
        }
        return darr;
    }

    snprintf(tmp, sizeof(tmp), "%s/%06d/%s/%06d/%s.txt.gz",
        filename, timestate, metadata->MeshForVar(varname).c_str(), domain, varname);
    visit_ifstream vfile(tmp);
    while (!vfile().eof())
    {
        int i, eIdx;
        char *p, vkey[32];
        double val = 0;

        vfile().getline(line, sizeof(line));
        if (vfile().eof()) break;

        p = line;
        sscanhdfs(p, "%s", vkey); // changes ',' to ' '
        eIdx = eKeyMap[vkey];
        p = strchr(p, ' ')+1;
        i = 0;
        while (i < ncomps)
        {
            switch (dtyp)
            {
                case VTK_CHAR: {char v; sscanhdfs(p, "%hhd", &v); val=(double)v; break;};
                case VTK_SHORT: {short v; sscanhdfs(p, "%hd", &v); val=(double)v; break;};
                case VTK_INT: {int v; sscanhdfs(p, "%d", &v); val=(double)v; break;};
                case VTK_LONG: {long v; sscanhdfs(p, "%ld", &v); val=(double)v; break;};
                case VTK_LONG_LONG: {long long v; sscanhdfs(p, "%lld", &v); val=(double)v; break;};
                case VTK_FLOAT: {float v; sscanhdfs(p, "%g", &v); val=(double)v; break;};
                case VTK_DOUBLE: {double v; sscanhdfs(p, "%lg", &v); val=(double)v; break;};
            }
            darr->SetComponent(eIdx, i, val);
            i++;
            p = strchr(p, ' ')+1;
        }
    }
    return darr;
}


// ****************************************************************************
//  Method: avtHDFSFileFormat::GetVectorVar
//
//  Purpose:
//      Gets a vector variable associated with this file.  Although VTK has
//      support for many different types, the best bet is vtkFloatArray, since
//      that is supported everywhere through VisIt.
//
//  Arguments:
//      timestate  The index of the timestate.  If GetNTimesteps returned
//                 'N' time steps, this is guaranteed to be between 0 and N-1.
//      domain     The index of the domain.  If there are NDomains, this
//                 value is guaranteed to be between 0 and NDomains-1,
//                 regardless of block origin.
//      varname    The name of the variable requested.
//
//  Programmer: miller86 -- generated by xml2avt
//  Creation:   Tue Apr 25 16:33:45 PST 2017
//
// ****************************************************************************

vtkDataArray *
avtHDFSFileFormat::GetVectorVar(int timestate, int domain,const char *varname)
{
    return GetVar(timestate, domain, varname);
}
